****************************************
*        *********					   *
* This do-file converts and appends    *
* files from from the spatial join     *
* process into a .dta usable in stata  *
****************************************


clear all
set more off

// Working directories
cd "$workpath"

// Locals we need for looping across years and IO directories
local years "2001 2003 2005 2007 2009 2011 2013 2017"
// 2019
local input_path "/Volumes/LaCie4TB/_EXTERNAL_WORKSPACE/crsh_land_use/data_march2024/data_raw"
local output_path "/Volumes/LaCie4TB/_EXTERNAL_WORKSPACE/crsh_land_use/data_march2024/data_processed"
local temp_path "/Volumes/LaCie4TB/_EXTERNAL_WORKSPACE/crsh_land_use/data_march2024/temp"



exit

***********CONVERTING AND APPENDING PARCELS FROM SPATIAL JOIN *********
* a. From shapefiles to dta files : parcels polygons
cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\parcels"
local shape : dir . files"*.shp"          				/*Getting all the .shp files il a local variable for the loop*/
local filename : subinstr local shape ".shp" "", all    /*removing the ".shp" extensions leaving just the file names*/

foreach file in `filename' {							/*looping over all the files in the local defined above*/
 spshape2dta `file'.shp,replace							/*converting .shp into .dta*/
 use `file'.dta, clear									/*opening the .dta file*/
 tostring centidsup polyidsup withidsup, replace		/*Assigning string format to some variables to avoid missing during appending processes*/
 destring *area* *neigh*, replace
 replace withidsup="" if withidsup=="."
 capture gen state_id = "`file'"						/*generating a variable to be used for merging process*/
 replace state_id=subinstr(state_id,"__","",1)
 capture gen polygon="parcel"							/*generating a variable to identify polygon type*/
 ren HubDist centdist									/*Renaming  the distance between the address-point and its assigned polygon from the centroid method*/
 ren HubDist_2 polydist /*Renaming  the distance between the address-point and its assigned polygon from the nearest polygon method*/
 keep adressid lon_g1 lat_g1 scottsid scotts_add acc_g1 year centdist polydist centidsup centarea_c centarea_d polyidsup polyarea_c polyarea_d withidsup witharea_c witharea_d polyneighb centneighb withneighb polygon state_id  /*Keeping only important variables*/
 replace accuracy="ROOFTOP" if accuracy==""
 keep adressid lon lat accuracy centdist polydist centidsup centarea_c centarea_d polyidsup polyarea_c polyarea_d withidsup witharea_c witharea_d polyneighb centneighb withneighb polygon state_id  /*Keeping only important variables*/
 order _all, alphabetic									/*ordering variables to avoid confusion during the appending processes*/
 sleep 500                       						/*Avoiding such Error as .... "file is read-only and cannot be modified or erase". This command is meant to give a delay to the computer before performing the next command */
 save `file'.dta, replace								/*Saving the converted .dta file*/
 erase `file'_shp.dta									/*Erasing the additional shapefile containing centroids, that have been generated during the convertion process*/
}


* b. Appending files with fs function
*cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\parcels"
cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\parcels"
capture erase "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\parcels\scotts_p.dta"
clear
fs *.dta												/*Using the fs function to append automatically all the .dta files contained in the current directory*/
append using `r(files)'

save "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\parcels\scotts_p.dta", replace


***********CONVERTING AND APPENDING BUILDINGS FROM SPATIAL JOIN*********
* a. From shapefiles to dta files
cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\buildings"
local shape : dir . files"*.shp"          				/*Getting all the .shp files*/
local filename : subinstr local shape ".shp" "", all    /*removing the ".shp" extensions leaving just the file names*/

foreach file in `filename' {							/*looping over all the files in the local defined above*/
 spshape2dta `file'.shp,replace							/*converting .shp into .dta*/
 use `file'.dta, clear									/*opening the .dta file*/
 tostring centidsup polyidsup withidsup, replace		/*Assigning string format to some variables to avoid missing during appending processes*/
 destring *area* *neigh*, replace
 replace withidsup="" if withidsup=="."
 replace polyneighb=0 if polyneighb==.
 replace centneighb=0 if centneighb==.
 replace withneighb=0 if withidsup!=""
 capture gen state_id = "`file'"						/*generating a variable to be used for merging process*/
 replace state_id=subinstr(state_id,"__","",1)
 capture gen polygon="building"							/*generating a variable to identify polygon type*/
 ren HubDist centdist									/*Renaming  the distance between the address-point and its assigned polygon from the centroid method*/
 ren HubDist_2 polydist 								/*Renaming  the distance between the address-point and its assigned polygon from the nearest polygon method*/
 replace accuracy="ROOFTOP" if accuracy==""
 keep adressid lon lat accuracy centdist polydist centidsup centarea_c centarea_d polyidsup polyarea_c polyarea_d withidsup witharea_c witharea_d polyneighb centneighb withneighb polygon state_id  /*Keeping only important variables*/
 order _all, alphabetic									/*ordering variables to avoid confusion during the appending processes*/
 sleep 500                       						/*Avoiding such error as .... "file is read-only and cannot be modified or erase". This command is meant to give a delay to the computer before performing the next command */
 save `file'.dta, replace								/*Saving the converted .dta file*/
 erase `file'_shp.dta									/*Erasing the additional shapefile containing centroids, that have been generated during the convertion process*/
}


* b. Appending files with fs function
cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\buildings"
capture erase "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\buildings\scotts_b.dta"
clear
fs *.dta												/*Using the fs function to append automatically all the .dta files contained in the current directory*/
append using `r(files)', force

save "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\buildings\scotts_b.dta", replace /*Saving the appended file*/


***********CONVERTING AND APPENDING HEIGHTS FROM SPATIAL JOIN*********
* a. From shapefiles to dta files
cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\heights"
local shape : dir . files"*.shp"          				/*Getting all the .shp files*/
local filename : subinstr local shape ".shp" "", all    /*re*/

foreach file in `filename' {							/*looping over all the files in the local defined above*/
 spshape2dta `file'.shp,replace							/*converting .shp into .dta*/
 use `file'.dta, clear									/*opening the .dta file*/
 tostring centidsup polyidsup withidsup, replace		/*Assigning string format to some variables to avoid missing during appending processes*/
 destring *area* *neigh*, replace
 replace withidsup="" if withidsup=="."
 replace polyneighb=0 if polyneighb==.
 replace centneighb=0 if centneighb==.
 replace withneighb=0 if withidsup!=""
 capture gen state_id = "`file'"						/*generating a variable to be used for merging process*/
 replace state_id=subinstr(state_id,"__","",1)
 capture gen polygon="height"							/*generating a variable to identify polygon outline origine*/
 ren HubDist centdist									/*Renaming  the distance between the address-point and its assigned polygon from the centroid method*/
 ren HubDist_2 polydist									/*Renaming  the distance between the address-point and its assigned polygon from the nearest polygon method*/
 replace accuracy="ROOFTOP" if accuracy==""
 egen polyBheight=rowmax(polyheig_1 polyheight)
 egen withBheight=rowmax(withheig_1 withheight)
 egen centBheight=rowmax(centheig_1 centheight)
 drop polyheight centheight withheight polyheig_1 centheig_1 withheig_1
 ren polyBheight polyheight
 ren centBheight centheight
 ren withBheight withheight
 keep adressid lon lat *heig* accuracy centdist polydist centidsup centarea_c centarea_d polyidsup polyarea_c polyarea_d withidsup witharea_c witharea_d polyneighb centneighb withneighb polygon state_id  /*Keeping only important variables*/
 destring adressid, replace
 order _all, alphabetic									/*ordering variables to avoid confusion during the appending processes*/
 sleep 500                       						/*Avoiding Error such as .... file is read-only and cannot be modified or erase. This is meant to give a delay to the computer before performing the next command */
 save `file'.dta, replace								/*Saving the converted .dta file*/
 erase `file'_shp.dta									/*Erasing the additional shapefile containing centroids, that have been generated during the convertion process*/
}

* b. Appending files with fs function
cd "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\heights"
capture erase "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\heights\scotts_h.dta"
clear
fs *.dta												/*Using the fs function to append automatically all the .dta files contained in the current directory*/
append using `r(files)'

save "C:\Users\ag191004\OneDrive - UQAM\DataBase\merged_data\heights\scotts_h.dta", replace /*Saving the appended file*/


***********CONVERTING AND APPENDING PARCELS JOINED WITH REGISTRARY DATA FROM SPATIAL JOIN *********
* a. From shapefiles to dta files : parcels polygons
cd "C:\Users\Théophile\OneDrive - UQAM (1)\DataBase\chapitre1\merged_data\registraire_qc"
local shape : dir . files"*.shp"          				/*Getting all the .shp files il a local variable for the loop*/
local filename : subinstr local shape ".shp" "", all    /*removing the ".shp" extensions leaving just the file names*/

foreach file in `filename' {							/*looping over all the files in the local defined above*/
 spshape2dta `file'.shp,replace							/*converting .shp into .dta*/
 use `file'.dta, clear									/*opening the .dta file*/
 tostring centidsup polyidsup withidsup, replace		/*Assigning string format to some variables to avoid missing during appending processes*/
 destring *area* *neigh*, replace
 replace withidsup="" if withidsup=="."
 capture gen state_id = "`file'"						/*generating a variable to be used for merging process*/
 replace state_id=subinstr(state_id,"__","",1)
 capture gen polygon="parcel"							/*generating a variable to identify polygon type*/
 ren HubDist centdist									/*Renaming  the distance between the address-point and its assigned polygon from the centroid method*/
 ren HubDist_2 polydist /*Renaming  the distance between the address-point and its assigned polygon from the nearest polygon method*/
 *keep adressid accuracy centdist polydist centidsup centarea_c centarea_d polyidsup polyarea_c polyarea_d withidsup witharea_c witharea_d polyneighb centneighb withneighb polygon state_id  /*Keeping only important variables*/
 replace accuracy="ROOFTOP" if accuracy==""
 keep lon lat adressid accuracy centdist polydist centidsup centarea_c centarea_d polyidsup polyarea_c polyarea_d withidsup witharea_c witharea_d polyneighb centneighb withneighb polygon state_id  /*Keeping only important variables*/
 order _all, alphabetic									/*ordering variables to avoid confusion during the appending processes*/
 sleep 500                       						/*Avoiding such Error as .... "file is read-only and cannot be modified or erase". This command is meant to give a delay to the computer before performing the next command */
 save `file'.dta, replace								/*Saving the converted .dta file*/
 erase `file'_shp.dta									/*Erasing the additional shapefile containing centroids, that have been generated during the convertion process*/
}


* b. Appending files with fs function
cd "C:\Users\Théophile\OneDrive - UQAM (1)\DataBase\chapitre1\merged_data\registraire_qc"
capture erase "C:\Users\Théophile\OneDrive - UQAM (1)\DataBase\chapitre1\merged_data\registraire_qc\registraire_p.dta"
capture erase "C:\Users\Théophile\OneDrive - UQAM (1)\DataBase\chapitre1\merged_data\registraire_qc\registraire_parcels.dta"

clear
fs *.dta												/*Using the fs function to append automatically all the .dta files contained in the current directory*/
append using `r(files)'

save "C:\Users\Théophile\OneDrive - UQAM (1)\DataBase\chapitre1\merged_data\registraire_qc\registraire_p.dta", replace
